import requests
from lxml import etree

headers = {
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36"}
url = "https://www.tvmao.com/drama/XGJiaHIl/episode/0-1"
domain = "https://www.tvmao.com"


response = requests.get(url=url, headers=headers)
# print(response.content.decode())
html = etree.HTML(response.content.decode())

with open("djzm.txt", "w", encoding='utf-8') as f:
    for li in html.xpath('//div[contains(@class,"epipage")]/ul/li'):
        # print(li.xpath('./a/@href')[0])
        page_url = li.xpath('./a/@href')[0]
        page_url = domain + page_url
        print(page_url)
        page_response = requests.get(url=page_url, headers=headers)
        page_html = etree.HTML(page_response.content.decode())
        article = page_html.xpath("//article/p/text()")
        for a in article:
            # print(a)
            f.write(a)
